{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "73876402-ebe0-4079-88a7-9ac6b7dbaae9",
   "metadata": {},
   "source": [
    "# Embed structurizr example dsl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3df2594d-d25d-4f3b-937a-d3fab1f303e6",
   "metadata": {},
   "source": [
    "## Create Embedding Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d8b8839e-6263-42ae-9ea0-ff2c3585d27b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Please define the following env variables in the .env file \n",
    "# \n",
    "# ollama_url = http://ollama:11434\n",
    "# embedding_model=nomic-embed-text\n",
    "# log_level=debug\n",
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "08dda744-1c51-4525-9a4b-5d11523c4236",
   "metadata": {},
   "outputs": [],
   "source": [
    "import utility\n",
    "embeddings = utility.get_embeddings()\n",
    "\n",
    "# from langchain.embeddings.ollama import OllamaEmbeddings\n",
    "\n",
    "# embeddings = OllamaEmbeddings(base_url=os.getenv(\"ollama_url\"), model=os.getenv(\"embedding_model\"), num_thread=os.getenv(\"num_thread\"))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e70f15a9-75b4-4b30-be76-26ffb106565a",
   "metadata": {},
   "source": [
    "### Create LanceDB table and Vector Store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fd9fe723-6f9e-4549-8258-037127c59b69",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 247 ms, sys: 37.9 ms, total: 285 ms\n",
      "Wall time: 876 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# https://lancedb.github.io/lancedb/guides/tables/#creating-empty-table\n",
    "import pyarrow as pa\n",
    "import lancedb\n",
    "from langchain_community.vectorstores import LanceDB\n",
    "\n",
    "db = lancedb.connect(\"lancedb\")\n",
    "## create table with overwrite mode\n",
    "example = db.create_table(\"example\", data = [{\"vector\":embeddings.embed_query(\"example\"), \"text\":\"example\"}], mode = \"overwrite\")\n",
    "\n",
    "## connect to the existing table\n",
    "# example = db.open_table('example')\n",
    "\n",
    "example_vectorstore = LanceDB(example, embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "712e2883-d8de-402b-a871-8693fa446d7c",
   "metadata": {},
   "source": [
    "## Read language reference markdown document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c00c72a6-dbac-4397-a9d5-03c47e9b1f0c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 13/13 [00:00<00:00, 1934.23it/s]\n"
     ]
    }
   ],
   "source": [
    "from langchain_community.document_loaders import DirectoryLoader, TextLoader\n",
    "\n",
    "loader = DirectoryLoader('structurizr/dsl/examples', glob=\"**/*.dsl\", show_progress=True, use_multithreading=True, loader_cls=TextLoader)\n",
    "\n",
    "example_workspaces = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "acbb5e39-0d45-4430-a2a2-a5ae3c1c9f03",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['structurizr/dsl/examples/microservices/workspace.dsl', 'structurizr/dsl/examples/system-catalog/dsl/catalog.dsl', 'structurizr/dsl/examples/system-catalog/dsl/customer-service/workspace.dsl', 'structurizr/dsl/examples/system-catalog/dsl/system-landscape/workspace.dsl', 'structurizr/dsl/examples/system-catalog/dsl/order-service/workspace.dsl', 'structurizr/dsl/examples/system-catalog/dsl/invoice-service/workspace.dsl', 'structurizr/dsl/examples/getting-started/workspace.dsl', 'structurizr/dsl/examples/big-bank-plc/workspace.dsl', 'structurizr/dsl/examples/big-bank-plc/model.dsl', 'structurizr/dsl/examples/big-bank-plc/internet-banking-system/workspace.dsl', 'structurizr/dsl/examples/amazon-web-services/workspace.dsl', 'structurizr/dsl/examples/big-bank-plc/system-landscape/workspace.dsl', 'structurizr/dsl/examples/financial-risk-system/workspace.dsl']\n"
     ]
    }
   ],
   "source": [
    "utility.debug([w.metadata[\"source\"] for w in example_workspaces])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c40e67c6-8537-473c-aa82-fe19097518f2",
   "metadata": {},
   "source": [
    "### Embed markdown sections into vector store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "37270d28-83d6-4fca-b7a3-a4c30d18682d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 27.5 ms, sys: 14.7 ms, total: 42.2 ms\n",
      "Wall time: 19.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "result = example_vectorstore.add_documents(example_workspaces)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05bc86a0-772b-4ab9-8605-096d50bbe84b",
   "metadata": {},
   "source": [
    "### Similarity Search - Vectorstore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b22abedf-624b-4934-9f6b-b7ff66ee0601",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "page_content='workspace \"Financial Risk System\" \"This is a simple (incomplete) example C4 model based upon the financial risk system architecture kata, which can be found at http://bit.ly/sa4d-risksystem\" {\\n\\n    model {\\n        businessUser = person \"Business User\" \"A regular business user.\"\\n        configurationUser = person \"Configuration User\" \"A regular business user who can also configure the parameters used in the risk calculations.\"\\n\\n        financialRiskSystem = softwareSystem \"Financial Risk System\" \"Calculates the bank\\'s exposure to risk for product X.\" \"Financial Risk System\"  {\\n            !docs docs\\n            !adrs adrs\\n        }\\n\\n        tradeDataSystem = softwareSystem \"Trade Data System\" \"The system of record for trades of type X.\"\\n        referenceDataSystem = softwareSystem \"Reference Data System\" \"Manages reference data for all counterparties the bank interacts with.\"\\n        referenceDataSystemV2 = softwareSystem \"Reference Data System v2.0\" \"Manages reference data for all counterparties the bank interacts with.\" \"Future State\"\\n        emailSystem = softwareSystem \"E-mail system\" \"The bank\\'s Microsoft Exchange system.\"\\n        centralMonitoringService = softwareSystem \"Central Monitoring Service\" \"The bank\\'s central monitoring and alerting dashboard.\"\\n        activeDirectory = softwareSystem \"Active Directory\" \"The bank\\'s authentication and authorisation system.\"\\n\\n        businessUser -> financialRiskSystem \"Views reports using\"\\n        financialRiskSystem -> tradeDataSystem \"Gets trade data from\"\\n        financialRiskSystem -> referenceDataSystem \"Gets counterparty data from\"\\n        financialRiskSystem -> referenceDataSystemV2 \"Gets counterparty data from\" \"\" \"Future State\"\\n        configurationUser -> financialRiskSystem \"Configures parameters using\"\\n        financialRiskSystem -> emailSystem \"Sends a notification that a report is ready to\"\\n        emailSystem -> businessUser \"Sends a notification that a report is ready to\" \"E-mail message\" \"Asynchronous\"\\n        financialRiskSystem -> centralMonitoringService \"Sends critical failure alerts to\" \"SNMP\" \"Asynchronous, Alert\"\\n        financialRiskSystem -> activeDirectory \"Uses for user authentication and authorisation\"\\n    }\\n         \\n    views {\\n\\n        systemContext financialRiskSystem \"Context\" \"An example System Context diagram for the Financial Risk System architecture kata.\" {\\n            include *\\n            autoLayout\\n        }\\n\\n        styles {\\n            element \"Element\" {\\n                color #ffffff\\n            }\\n            element \"Software System\" {\\n                background #801515\\n                shape RoundedBox\\n            }\\n            element \"Financial Risk System\" {\\n                background #550000\\n                color #ffffff\\n            }\\n            element \"Future State\" {\\n                opacity 30\\n            }\\n            element \"Person\" {\\n                background #d46a6a\\n                shape Person\\n            }\\n            relationship \"Relationship\" {\\n                dashed false\\n            }\\n            relationship \"Asynchronous\" {\\n                dashed true\\n            }\\n            relationship \"Alert\" {\\n                color #ff0000\\n            }\\n            relationship \"Future State\" {\\n                opacity 30\\n            }\\n       }\\n\\n    }\\n}' metadata={'source': 'structurizr/dsl/examples/financial-risk-system/workspace.dsl'}\n"
     ]
    }
   ],
   "source": [
    "utility.debug(example_workspaces[12])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "718c818d-c6df-44a0-8a0e-b795a4f33556",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "workspace \"Financial Risk System\" \"This is a simple (incomplete) example C4 model based upon the financial risk system architecture kata, which can be found at http://bit.ly/sa4d-risksystem\" {\n",
      "\n",
      "    model {\n",
      "        businessUser = person \"Business User\" \"A regular business user.\"\n",
      "        configurationUser = person \"Configuration User\" \"A regular business user who can also configure the parameters used in the risk calculations.\"\n",
      "\n",
      "        financialRiskSystem = softwareSystem \"Financial Risk System\" \"Calculates the bank's exposure to risk for product X.\" \"Financial Risk System\"  {\n",
      "            !docs docs\n",
      "            !adrs adrs\n",
      "        }\n",
      "\n",
      "        tradeDataSystem = softwareSystem \"Trade Data System\" \"The system of record for trades of type X.\"\n",
      "        referenceDataSystem = softwareSystem \"Reference Data System\" \"Manages reference data for all counterparties the bank interacts with.\"\n",
      "        referenceDataSystemV2 = softwareSystem \"Reference Data System v2.0\" \"Manages reference data for all counterparties the bank interacts with.\" \"Future State\"\n",
      "        emailSystem = softwareSystem \"E-mail system\" \"The bank's Microsoft Exchange system.\"\n",
      "        centralMonitoringService = softwareSystem \"Central Monitoring Service\" \"The bank's central monitoring and alerting dashboard.\"\n",
      "        activeDirectory = softwareSystem \"Active Directory\" \"The bank's authentication and authorisation system.\"\n",
      "\n",
      "        businessUser -> financialRiskSystem \"Views reports using\"\n",
      "        financialRiskSystem -> tradeDataSystem \"Gets trade data from\"\n",
      "        financialRiskSystem -> referenceDataSystem \"Gets counterparty data from\"\n",
      "        financialRiskSystem -> referenceDataSystemV2 \"Gets counterparty data from\" \"\" \"Future State\"\n",
      "        configurationUser -> financialRiskSystem \"Configures parameters using\"\n",
      "        financialRiskSystem -> emailSystem \"Sends a notification that a report is ready to\"\n",
      "        emailSystem -> businessUser \"Sends a notification that a report is ready to\" \"E-mail message\" \"Asynchronous\"\n",
      "        financialRiskSystem -> centralMonitoringService \"Sends critical failure alerts to\" \"SNMP\" \"Asynchronous, Alert\"\n",
      "        financialRiskSystem -> activeDirectory \"Uses for user authentication and authorisation\"\n",
      "    }\n",
      "         \n",
      "    views {\n",
      "\n",
      "        systemContext financialRiskSystem \"Context\" \"An example System Context diagram for the Financial Risk System architecture kata.\" {\n",
      "            include *\n",
      "            autoLayout\n",
      "        }\n",
      "\n",
      "        styles {\n",
      "            element \"Element\" {\n",
      "                color #ffffff\n",
      "            }\n",
      "            element \"Software System\" {\n",
      "                background #801515\n",
      "                shape RoundedBox\n",
      "            }\n",
      "            element \"Financial Risk System\" {\n",
      "                background #550000\n",
      "                color #ffffff\n",
      "            }\n",
      "            element \"Future State\" {\n",
      "                opacity 30\n",
      "            }\n",
      "            element \"Person\" {\n",
      "                background #d46a6a\n",
      "                shape Person\n",
      "            }\n",
      "            relationship \"Relationship\" {\n",
      "                dashed false\n",
      "            }\n",
      "            relationship \"Asynchronous\" {\n",
      "                dashed true\n",
      "            }\n",
      "            relationship \"Alert\" {\n",
      "                color #ff0000\n",
      "            }\n",
      "            relationship \"Future State\" {\n",
      "                opacity 30\n",
      "            }\n",
      "       }\n",
      "\n",
      "    }\n",
      "}\n",
      "CPU times: user 8.33 ms, sys: 3.02 ms, total: 11.3 ms\n",
      "Wall time: 82.5 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "result = example_vectorstore.similarity_search(\"Financial Risk System\")\n",
    "# for doc in result:\n",
    "#     print(f'*******\\n{doc.page_content}\\n', flush=True)\n",
    "\n",
    "utility.debug(result[0].page_content)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf13d268-c8cd-48a4-b50c-22789aefc6f2",
   "metadata": {},
   "source": [
    "### Similarity Search - Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "14ae9e81-87d2-4bd3-b909-05db2b874de0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.84 ms, sys: 4.86 ms, total: 13.7 ms\n",
      "Wall time: 78.1 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vector</th>\n",
       "      <th>text</th>\n",
       "      <th>_distance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[1.4288399, 0.9221773, -1.6998386, -0.8545214,...</td>\n",
       "      <td>workspace \"Financial Risk System\" \"This is a s...</td>\n",
       "      <td>346.203033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0.31328577, 2.1217017, -2.2669632, -1.649383,...</td>\n",
       "      <td>// https://structurizr.com/share/28201\\n\\nwork...</td>\n",
       "      <td>529.519958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[1.267492, 0.8016706, -1.7545884, -0.5837192, ...</td>\n",
       "      <td>/*\\n * This is a combined version of the follo...</td>\n",
       "      <td>532.953918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[0.793906, 1.3044648, -2.2495568, -0.86547333,...</td>\n",
       "      <td>// https://structurizr.com/share/36141\\n\\nwork...</td>\n",
       "      <td>536.596924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0.6189178, 2.25288, -1.3907903, -0.9112563, 0...</td>\n",
       "      <td>workspace {\\n\\n    model {\\n        customer =...</td>\n",
       "      <td>560.789795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[-0.09873457, 1.2760388, -2.7166872, -1.344648...</td>\n",
       "      <td>workspace \"Amazon Web Services Example\" \"An ex...</td>\n",
       "      <td>566.364929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[0.33687425, 1.1372119, -1.5293717, -0.799164,...</td>\n",
       "      <td>workspace {\\n    \\n    !identifiers hierarchic...</td>\n",
       "      <td>583.295471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[-0.15960225, 1.6412851, -1.5648005, -1.167354...</td>\n",
       "      <td>workspace extends ../catalog.dsl {\\n\\n    mode...</td>\n",
       "      <td>600.112427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[0.23624358, 1.3600173, -2.067849, -1.8004003,...</td>\n",
       "      <td>workspace extends ../catalog.dsl {\\n\\n    !plu...</td>\n",
       "      <td>604.471863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[0.07546459, 1.3151995, -1.4226674, -1.551525,...</td>\n",
       "      <td>workspace extends ../catalog.dsl {\\n\\n    mode...</td>\n",
       "      <td>605.560669</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              vector  \\\n",
       "0  [1.4288399, 0.9221773, -1.6998386, -0.8545214,...   \n",
       "1  [0.31328577, 2.1217017, -2.2669632, -1.649383,...   \n",
       "2  [1.267492, 0.8016706, -1.7545884, -0.5837192, ...   \n",
       "3  [0.793906, 1.3044648, -2.2495568, -0.86547333,...   \n",
       "4  [0.6189178, 2.25288, -1.3907903, -0.9112563, 0...   \n",
       "5  [-0.09873457, 1.2760388, -2.7166872, -1.344648...   \n",
       "6  [0.33687425, 1.1372119, -1.5293717, -0.799164,...   \n",
       "7  [-0.15960225, 1.6412851, -1.5648005, -1.167354...   \n",
       "8  [0.23624358, 1.3600173, -2.067849, -1.8004003,...   \n",
       "9  [0.07546459, 1.3151995, -1.4226674, -1.551525,...   \n",
       "\n",
       "                                                text   _distance  \n",
       "0  workspace \"Financial Risk System\" \"This is a s...  346.203033  \n",
       "1  // https://structurizr.com/share/28201\\n\\nwork...  529.519958  \n",
       "2  /*\\n * This is a combined version of the follo...  532.953918  \n",
       "3  // https://structurizr.com/share/36141\\n\\nwork...  536.596924  \n",
       "4  workspace {\\n\\n    model {\\n        customer =...  560.789795  \n",
       "5  workspace \"Amazon Web Services Example\" \"An ex...  566.364929  \n",
       "6  workspace {\\n    \\n    !identifiers hierarchic...  583.295471  \n",
       "7  workspace extends ../catalog.dsl {\\n\\n    mode...  600.112427  \n",
       "8  workspace extends ../catalog.dsl {\\n\\n    !plu...  604.471863  \n",
       "9  workspace extends ../catalog.dsl {\\n\\n    mode...  605.560669  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "example.search(embeddings.embed_query(\"Financial Risk System\")) \\\n",
    "    .limit(10) \\\n",
    "    .nprobes(20) \\\n",
    "    .refine_factor(10) \\\n",
    "    .to_pandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db83dc0f-e28b-4370-8f72-b2e1abe39a7e",
   "metadata": {},
   "source": [
    "### Similarity Search - Retriever"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "5ff4715b-697a-4991-bf3d-ecb1748a754e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# retriever = language_vectorstore.as_retriever()\n",
    "retriever = example_vectorstore.as_retriever(search_kwargs={\"k\": 1})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "955f8a00-7fa2-4ca0-ab76-145eb8da085a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['workspace \"Financial Risk System\" \"This is a simple (incomplete) example C4 model based upon the financial risk system architecture kata, which can be found at http://bit.ly/sa4d-risksystem\" {\\n\\n    model {\\n        businessUser = person \"Business User\" \"A regular business user.\"\\n        configurationUser = person \"Configuration User\" \"A regular business user who can also configure the parameters used in the risk calculations.\"\\n\\n        financialRiskSystem = softwareSystem \"Financial Risk System\" \"Calculates the bank\\'s exposure to risk for product X.\" \"Financial Risk System\"  {\\n            !docs docs\\n            !adrs adrs\\n        }\\n\\n        tradeDataSystem = softwareSystem \"Trade Data System\" \"The system of record for trades of type X.\"\\n        referenceDataSystem = softwareSystem \"Reference Data System\" \"Manages reference data for all counterparties the bank interacts with.\"\\n        referenceDataSystemV2 = softwareSystem \"Reference Data System v2.0\" \"Manages reference data for all counterparties the bank interacts with.\" \"Future State\"\\n        emailSystem = softwareSystem \"E-mail system\" \"The bank\\'s Microsoft Exchange system.\"\\n        centralMonitoringService = softwareSystem \"Central Monitoring Service\" \"The bank\\'s central monitoring and alerting dashboard.\"\\n        activeDirectory = softwareSystem \"Active Directory\" \"The bank\\'s authentication and authorisation system.\"\\n\\n        businessUser -> financialRiskSystem \"Views reports using\"\\n        financialRiskSystem -> tradeDataSystem \"Gets trade data from\"\\n        financialRiskSystem -> referenceDataSystem \"Gets counterparty data from\"\\n        financialRiskSystem -> referenceDataSystemV2 \"Gets counterparty data from\" \"\" \"Future State\"\\n        configurationUser -> financialRiskSystem \"Configures parameters using\"\\n        financialRiskSystem -> emailSystem \"Sends a notification that a report is ready to\"\\n        emailSystem -> businessUser \"Sends a notification that a report is ready to\" \"E-mail message\" \"Asynchronous\"\\n        financialRiskSystem -> centralMonitoringService \"Sends critical failure alerts to\" \"SNMP\" \"Asynchronous, Alert\"\\n        financialRiskSystem -> activeDirectory \"Uses for user authentication and authorisation\"\\n    }\\n         \\n    views {\\n\\n        systemContext financialRiskSystem \"Context\" \"An example System Context diagram for the Financial Risk System architecture kata.\" {\\n            include *\\n            autoLayout\\n        }\\n\\n        styles {\\n            element \"Element\" {\\n                color #ffffff\\n            }\\n            element \"Software System\" {\\n                background #801515\\n                shape RoundedBox\\n            }\\n            element \"Financial Risk System\" {\\n                background #550000\\n                color #ffffff\\n            }\\n            element \"Future State\" {\\n                opacity 30\\n            }\\n            element \"Person\" {\\n                background #d46a6a\\n                shape Person\\n            }\\n            relationship \"Relationship\" {\\n                dashed false\\n            }\\n            relationship \"Asynchronous\" {\\n                dashed true\\n            }\\n            relationship \"Alert\" {\\n                color #ff0000\\n            }\\n            relationship \"Future State\" {\\n                opacity 30\\n            }\\n       }\\n\\n    }\\n}']\n"
     ]
    }
   ],
   "source": [
    "result = retriever.get_relevant_documents(\"Financial Risk System\")\n",
    "\n",
    "utility.debug([doc.page_content for doc in result])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "01d295fb-045e-49f3-95f0-3289ea58a957",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['workspace \"Financial Risk System\" \"This is a simple (incomplete) example C4 model based upon the financial risk system architecture kata, which can be found at http://bit.ly/sa4d-risksystem\" {\\n\\n    model {\\n        businessUser = person \"Business User\" \"A regular business user.\"\\n        configurationUser = person \"Configuration User\" \"A regular business user who can also configure the parameters used in the risk calculations.\"\\n\\n        financialRiskSystem = softwareSystem \"Financial Risk System\" \"Calculates the bank\\'s exposure to risk for product X.\" \"Financial Risk System\"  {\\n            !docs docs\\n            !adrs adrs\\n        }\\n\\n        tradeDataSystem = softwareSystem \"Trade Data System\" \"The system of record for trades of type X.\"\\n        referenceDataSystem = softwareSystem \"Reference Data System\" \"Manages reference data for all counterparties the bank interacts with.\"\\n        referenceDataSystemV2 = softwareSystem \"Reference Data System v2.0\" \"Manages reference data for all counterparties the bank interacts with.\" \"Future State\"\\n        emailSystem = softwareSystem \"E-mail system\" \"The bank\\'s Microsoft Exchange system.\"\\n        centralMonitoringService = softwareSystem \"Central Monitoring Service\" \"The bank\\'s central monitoring and alerting dashboard.\"\\n        activeDirectory = softwareSystem \"Active Directory\" \"The bank\\'s authentication and authorisation system.\"\\n\\n        businessUser -> financialRiskSystem \"Views reports using\"\\n        financialRiskSystem -> tradeDataSystem \"Gets trade data from\"\\n        financialRiskSystem -> referenceDataSystem \"Gets counterparty data from\"\\n        financialRiskSystem -> referenceDataSystemV2 \"Gets counterparty data from\" \"\" \"Future State\"\\n        configurationUser -> financialRiskSystem \"Configures parameters using\"\\n        financialRiskSystem -> emailSystem \"Sends a notification that a report is ready to\"\\n        emailSystem -> businessUser \"Sends a notification that a report is ready to\" \"E-mail message\" \"Asynchronous\"\\n        financialRiskSystem -> centralMonitoringService \"Sends critical failure alerts to\" \"SNMP\" \"Asynchronous, Alert\"\\n        financialRiskSystem -> activeDirectory \"Uses for user authentication and authorisation\"\\n    }\\n         \\n    views {\\n\\n        systemContext financialRiskSystem \"Context\" \"An example System Context diagram for the Financial Risk System architecture kata.\" {\\n            include *\\n            autoLayout\\n        }\\n\\n        styles {\\n            element \"Element\" {\\n                color #ffffff\\n            }\\n            element \"Software System\" {\\n                background #801515\\n                shape RoundedBox\\n            }\\n            element \"Financial Risk System\" {\\n                background #550000\\n                color #ffffff\\n            }\\n            element \"Future State\" {\\n                opacity 30\\n            }\\n            element \"Person\" {\\n                background #d46a6a\\n                shape Person\\n            }\\n            relationship \"Relationship\" {\\n                dashed false\\n            }\\n            relationship \"Asynchronous\" {\\n                dashed true\\n            }\\n            relationship \"Alert\" {\\n                color #ff0000\\n            }\\n            relationship \"Future State\" {\\n                opacity 30\\n            }\\n       }\\n\\n    }\\n}']\n"
     ]
    }
   ],
   "source": [
    "result = retriever.get_relevant_documents(\"Reference Data System\")\n",
    "\n",
    "utility.debug([doc.page_content for doc in result])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78bd5f1a-8dbd-42f4-9fd2-16ac11841b1d",
   "metadata": {},
   "source": [
    "## Create code snippet vector store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "271f16cc-0313-499f-b15e-c416389468d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2024-03-24T10:40:39Z WARN  lance::dataset] No existing dataset at /home/jovyan/work/lancedb/codes.lance, it will be created\n"
     ]
    }
   ],
   "source": [
    "codes = db.create_table(\"codes\", data = [{\"vector\":embeddings.embed_query(\"code\"), \"text\":\"code\"}], mode = \"overwrite\")\n",
    "\n",
    "## connect to the existing table\n",
    "# example = db.open_table('example')\n",
    "\n",
    "codes_vectorstore = LanceDB(codes, embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8732cf11-106f-4680-b2ea-ace53453e0d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# example_workspaces[0].page_content"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f319734",
   "metadata": {},
   "source": [
    "### Recognize the entities in code snippets\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f240da2e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
